#!/bin/bash


model_path="/vepfs_train2/meichaoyang/model/qwen/Qwen2-7B-Instruct"


export WANDB_API_KEY=9afc62359e50f5d0b24fee88ce7ce8d162e998ed
export WANDB_DISABLED=true


echo "配置环境变量"
export NCCL_SOCKET_IFNAME=eth0
export NCCL_IB_GID_INDEX=3
export NCCL_IB_DISABLE=0
export NCCL_IB_HCA=mlx5_bond_0,mlx5_bond_1,mlx5_bond_2,mlx5_bond_3,mlx5_bond_4,mlx5_bond_5,mlx5_bond_6,mlx5_bond_7
export NCCL_NET_GDR_LEVEL=2
export NCCL_IB_QPS_PER_CONNECTION=4
export NCCL_IB_TC=160
export NCCL_IB_TIMEOUT=22


echo "配置训练参数"
# 配置训练数据集路径
dataset_dir="/vepfs_train2/meichaoyang/train_data"
dataset="alpaca_zh_demo,glaive_toolcall_zh_demo"

# 配置模型输出路径
output_dir="/vepfs_train2/meichaoyang/checkpoint/qwen2_7b_chat_full_sft_alpaca_zh_demo_glaive_toolcall_zh_demo_bs128_lr2e-6_1node_len20k"

# 配置超参
gradient_accumulation_steps=16  # batch size = gradient_accumulation_steps * per_device_train_batch_size
learning_rate=2e-6


echo "启动训练"
deepspeed --num_gpus 8 --master_port=9901 src/train.py \
    --deepspeed conf/ds_stage3_config_qwen_no_offload.json \
    --stage sft \
    --model_name_or_path $model_path \
    --do_train \
    --dataset_dir $dataset_dir \
    --dataset $dataset \
    --preprocessing_num_workers 16 \
    --template qwen \
    --finetuning_type full \
    --output_dir $output_dir \
    --overwrite_cache \
    --per_device_train_batch_size 1 \
    --gradient_accumulation_steps $gradient_accumulation_steps \
    --lr_scheduler_type cosine \
    --logging_steps 1 \
    --warmup_steps 0 \
    --save_steps 2000 \
    --save_only_model \
    --learning_rate $learning_rate \
    --num_train_epochs 3.0 \
    --plot_loss \
    --bf16 \
    --ignore_len 20480 \
    --cutoff_len 102400 \
    --flash_attn fa2 \
    --use_fast_tokenizer true
